package cn.cloud9.rdp;

import cn.cloud9.rdp.po.RegionCell;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.net.HttpCookie;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;

/**
 * @projectName: 行政区域爬取工具
 * @author: 戴知舟
 * @date: 2022年06月29日 09:55
 * @version: 1.0
 */
public class MainApplication {
    public static final int DELAY = 500;
    public static final String HEADER_URL = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/";
    public static final String COOKIE_KEY = "SF_cookie_1";
    public static final String COOKIE_HEADER_KEY = "Cookie";
    public static final String USER_AGENT = "User-Agent";
    public static final String[] BROWSER_AGENTS = {
        "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0"
    };
    public final static Random R = new Random();
    public static final String TABLE_NAME = "region2021";

    public static Map<String, String> getNewRequestHeader() {
        return new ConcurrentHashMap<String, String>(){{
            this.put("Host", "www.stats.gov.cn");
//            this.put("Server", "CWAP-waf");
//            this.put("Content-Length", "2351");
//            this.put("Content-Type", "text/html");
//            this.put("Accept-Ranges", "bytes");
//            this.put("Content-Encoding", "gzip");
//            this.put("ETag", "1736-5d4bccabedf00-gzip");
//            this.put("Vary", "Accept-Encoding");
//            this.put("X-Powered-By", "anyu.qianxin.com");
//            this.put("WZWS-RAY", "1129-1656502841.38-w-waf03cdm");
        }};
    }


    public static void main(String[] args) throws SQLException {
        Db db = Db.use();
        // 清空表记录
        int execute = db.execute("TRUNCATE TABLE region2021", null);

        // 第一层请求
        HttpRequest getRequest = HttpUtil.createGet(HEADER_URL);
        HttpResponse httpResponse = getRequest.execute();
        HttpCookie cookie = httpResponse.getCookie(COOKIE_KEY);
        System.out.println(cookie);

        // 取响应状态
        int status = httpResponse.getStatus();
        // 取响应data
        String body = httpResponse.body();
        System.out.println("省 响应状态 " + status);
        System.out.println("省 响应状态 " + body);
        if (status != 200) {
            System.out.println("爬取异常，程序终止");
            return;
        }

        final Document DOC = Jsoup.parse(body);
        Elements provinceTrList = DOC.getElementsByClass("provincetr");
        provinceTrList.forEach(tr -> {
            Elements provinceAList = tr.getElementsByTag("a");
            provinceAList.forEach(a -> {
                int provinceTrIndex = provinceTrList.indexOf(tr) + 1;
                int provinceALinkIndex = provinceAList.indexOf(a) + 1;

                RegionCell cell = RegionCell.builder()
                    .name(a.text())
                    .regionCode(String.valueOf(provinceTrIndex) + provinceALinkIndex)
                    .url(HEADER_URL + a.attr("href"))
                    .parentCode(String.valueOf(0))
                    .genTime(LocalDateTime.now())
                    .build();
                System.out.println(cell);

                // 插入省份
                try {
                    db.insert(Entity
                        .create(TABLE_NAME)
                        .set("CODE", cell.getRegionCode())
                        .set("PARENT_CODE", cell.getParentCode())
                        .set("NAME", cell.getName())
                        .set("LEVEL", 1)
                        .set("LINK", cell.getUrl())
                        .set("GEN_TIME", cell.getGenTime())
                    );
                } catch (SQLException e) {
                    e.printStackTrace();
                }

                // 请求之前缓一缓
                try {
                    Thread.sleep(DELAY);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                // 设置header头
                Map<String, String> header = getNewRequestHeader();
                header.put(COOKIE_HEADER_KEY, cookie.toString());
                header.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                HttpResponse response = HttpUtil.createGet(cell.getUrl()).addHeaders(header).execute();
                List<HttpCookie> provinceCookies = response.getCookies();
                int provinceStatus = response.getStatus();
                System.out.println(cell.getName() + " | " + cell.getUrl() + " | 请求状态" + provinceStatus);
                String provinceBody = response.body();
                final Document provinceDOC = Jsoup.parse(provinceBody);

                Elements cityTrList = provinceDOC.getElementsByClass("citytr");
                cityTrList.forEach(cityTr -> {

                    Elements cityTds = cityTr.getElementsByTag("td");
                    Element codeTd = cityTds.get(0);
                    Element nameTd = cityTds.get(1);
                    Elements aTagInCodeTd = codeTd.getElementsByTag("a");
                    Elements aTagInNameTd = nameTd.getElementsByTag("a");

                    // 取城乡代码
                    String regionCityCode;
                    String cityNextHref = null;

                    boolean isLinkTag = null != aTagInCodeTd;
                    if (isLinkTag) {
                        regionCityCode = aTagInCodeTd.text();
                        cityNextHref = aTagInCodeTd.attr("href");
                    } else
                        regionCityCode = codeTd.text();
                    String regionName;
                    if (null != aTagInNameTd)
                        regionName = aTagInNameTd.text();
                    else
                        regionName = nameTd.text();

                    RegionCell.RegionCellBuilder cellBuilder = RegionCell.builder();
                    if (isLinkTag)
                        cellBuilder.url(HEADER_URL + cityNextHref);

                    cellBuilder.name(regionName);
                    cellBuilder.regionCode(regionCityCode);
                    cellBuilder.parentCode(cell.getRegionCode());
                    cellBuilder.genTime(LocalDateTime.now()).build();
                    RegionCell cityCell = cellBuilder.build();
                    System.out.println(cityCell);

                    try {
                        db.insert(Entity
                            .create(TABLE_NAME)
                            .set("CODE", cityCell.getRegionCode())
                            .set("PARENT_CODE", cityCell.getParentCode())
                            .set("NAME", cityCell.getName())
                            .set("LEVEL", 2)
                            .set("LINK", cityCell.getUrl())
                            .set("GEN_TIME", cityCell.getGenTime())
                        );
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }

                    if (null == cityNextHref)
                        return;

                    // 请求之前缓一缓
                    try {
                        Thread.sleep(DELAY);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }

                    Map<String, String> countyHeader = getNewRequestHeader();
                    provinceCookies.forEach( provinceCookie -> countyHeader.put(COOKIE_HEADER_KEY, provinceCookie.toString() + ";"));
                    countyHeader.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                    HttpResponse countyResponse = HttpUtil.createGet(cityCell.getUrl()).addHeaders(countyHeader).execute();
                    List<HttpCookie> countyCookies = countyResponse.getCookies();
                    int countyStatus = countyResponse.getStatus();
                    String countyBody = countyResponse.body();
                    System.out.println(cityCell.getName() + " | " + cityCell.getUrl() + " | 请求状态" + countyStatus);
                    Document countyDOC = Jsoup.parse(countyBody);

                    Elements countyTrList = countyDOC.getElementsByClass("countytr");
                    countyTrList.forEach(countyTr -> {

                        Elements countyTds = countyTr.getElementsByTag("td");
                        Element countyCodeTd = countyTds.get(0);
                        Element countyNameTd = countyTds.get(1);
                        Elements aTagInCountyCodeTd = countyCodeTd.getElementsByTag("a");
                        Elements aTagInCountyNameTd = countyNameTd.getElementsByTag("a");
                        // 取城乡代码
                        String regionCountyCode;
                        String countyNextHref = null;

                        boolean isCountyLinkTag = null != aTagInCountyCodeTd;
                        if (isCountyLinkTag) {
                            regionCountyCode = aTagInCountyCodeTd.text();
                            countyNextHref = aTagInCountyCodeTd.attr("href");
                        } else
                            regionCountyCode = countyCodeTd.text();
                        String regionCountyName;
                        if (null != aTagInCountyNameTd)
                            regionCountyName = aTagInCountyNameTd.text();
                        else
                            regionCountyName = countyNameTd.text();

                        RegionCell.RegionCellBuilder countyCellBuilder = RegionCell.builder();
                        if (isCountyLinkTag) {
                            int index = countyNextHref.indexOf('/');
                            String provincePath = countyNextHref.substring(index + 1, index + 3) + "/";
                            String countyUrl = HEADER_URL + provincePath + countyNextHref;
                            countyCellBuilder.url(countyUrl);
                        }

                        countyCellBuilder.name(regionCountyName);
                        countyCellBuilder.regionCode(regionCountyCode);
                        countyCellBuilder.genTime(LocalDateTime.now());
                        countyCellBuilder.parentCode(cityCell.getRegionCode());
                        RegionCell countyCell = countyCellBuilder.build();
                        System.out.println(countyCell);


                        try {
                            db.insert(Entity
                                .create(TABLE_NAME)
                                .set("CODE", countyCell.getRegionCode())
                                .set("PARENT_CODE", countyCell.getParentCode())
                                .set("NAME", countyCell.getName())
                                .set("LEVEL", 3)
                                .set("LINK", countyCell.getUrl())
                                .set("GEN_TIME", countyCell.getGenTime())
                            );
                        } catch (SQLException e) {
                            e.printStackTrace();
                        }

                        if (null == countyNextHref)
                            return;

                        // 请求之前缓一缓
                        try {
                            Thread.sleep(DELAY);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }

                        Map<String, String> townHeader = getNewRequestHeader();
                        countyCookies.forEach(
                                provinceCookie -> townHeader.put(COOKIE_HEADER_KEY, provinceCookie.toString() + ";"));
                        townHeader.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                        HttpResponse townResponse = HttpUtil.createGet(countyCell.getUrl()).addHeaders(townHeader).execute();
                        List<HttpCookie> townCookies = townResponse.getCookies();
                        int townStatus = townResponse.getStatus();
                        String townBody = townResponse.body();
                        System.out.println(countyCell.getName() + " | " + countyCell.getUrl() + " | 请求状态" + townStatus);
                        Document townDOC = Jsoup.parse(townBody);
                        Elements townTrList = townDOC.getElementsByClass("towntr");
                        townTrList.forEach(townTr -> {

                            Elements townTds = townTr.getElementsByTag("td");
                            Element townCodeTd = townTds.get(0);
                            Element townNameTd = townTds.get(1);
                            Elements aTagInTownCodeTd = townCodeTd.getElementsByTag("a");
                            Elements aTagInTownNameTd = townNameTd.getElementsByTag("a");
                            // 取城乡代码
                            String regionTownCode;
                            String townNextHref = null;

                            boolean isTownLinkTag = null != aTagInTownCodeTd;
                            if (isTownLinkTag) {
                                regionTownCode = aTagInTownCodeTd.text();
                                townNextHref = aTagInTownCodeTd.attr("href");
                            } else
                                regionTownCode = townCodeTd.text();
                            String regionTownName;
                            if (null != aTagInTownNameTd)
                                regionTownName = aTagInTownNameTd.text();
                            else
                                regionTownName = townNameTd.text();

                            RegionCell.RegionCellBuilder townCellBuilder = RegionCell.builder();
                            if (isTownLinkTag) {
                                int index = townNextHref.indexOf("/");
                                String provincePath = townNextHref.substring(index + 1, index + 3);
                                String cityPath = "/" + townNextHref.substring(index + 3, index + 5) + "/";
                                String url = HEADER_URL + provincePath + cityPath + townNextHref;
                                townCellBuilder.url(url);
                            }

                            townCellBuilder.name(regionTownName);
                            townCellBuilder.regionCode(regionTownCode);
                            townCellBuilder.genTime(LocalDateTime.now());
                            townCellBuilder.parentCode(countyCell.getRegionCode());
                            RegionCell townCell = townCellBuilder.build();
                            System.out.println(townCell);

                            try {
                                db.insert(Entity
                                    .create(TABLE_NAME)
                                    .set("CODE", townCell.getRegionCode())
                                    .set("PARENT_CODE", townCell.getParentCode())
                                    .set("NAME", townCell.getName())
                                    .set("LEVEL", 4)
                                    .set("LINK", townCell.getUrl())
                                    .set("GEN_TIME", townCell.getGenTime())
                                );
                            } catch (SQLException e) {
                                e.printStackTrace();
                            }


                            if (null == townNextHref)
                                return;
                            // 请求之前缓一缓
                            try {
                                Thread.sleep(DELAY);
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                            Map<String, String> villageHeader = getNewRequestHeader();
                            townCookies.forEach(
                                    townCookie -> villageHeader.put(COOKIE_HEADER_KEY, townCookie.toString() + ";"));
                            villageHeader.put(USER_AGENT, BROWSER_AGENTS[R.nextInt(BROWSER_AGENTS.length)]);
                            HttpResponse villageResponse = HttpUtil.createGet(townCell.getUrl()).addHeaders(villageHeader).execute();
                            int villageStatus = villageResponse.getStatus();
                            String villageBody = villageResponse.body();
                            System.out.println(townCell.getName() + " | " + townCell.getUrl() + " | 请求状态" + villageStatus);
                            Document villageDOC = Jsoup.parse(villageBody);
                            Elements villageTrList = villageDOC.getElementsByClass("villagetr");

                            villageTrList.forEach(villageTr -> {
                                Elements villageTdList = villageTr.getElementsByTag("td");

                                Element villageCodeTd = villageTdList.get(0);
                                Element villageTypeCodeTd = villageTdList.get(1);
                                Element villageNameTd = villageTdList.get(2);

                                RegionCell.RegionCellBuilder villageCellBuilder = RegionCell.builder();
                                villageCellBuilder.regionCode(villageCodeTd.text());
                                villageCellBuilder.name(villageNameTd.text());
                                villageCellBuilder.typeCode(villageTypeCodeTd.text());
                                villageCellBuilder.parentCode(townCell.getRegionCode());
                                villageCellBuilder.genTime(LocalDateTime.now());
                                RegionCell villageCell = villageCellBuilder.build();
                                System.out.println(villageCell);

                                try {
                                    db.insert(Entity
                                        .create(TABLE_NAME)
                                        .set("CODE", villageCell.getRegionCode())
                                        .set("PARENT_CODE", villageCell.getParentCode())
                                        .set("NAME", villageCell.getName())
                                        .set("LEVEL", 3)
                                        .set("TYPE_CODE", villageCell.getTypeCode())
                                        .set("LINK", villageCell.getUrl())
                                        .set("GEN_TIME", villageCell.getGenTime())
                                    );
                                } catch (SQLException e) {
                                    e.printStackTrace();
                                }
                            });
                        });
                    });
                });
            });
        });
    }
}
